{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda2\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "C:\\ProgramData\\Anaconda2\\lib\\site-packages\\sklearn\\grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.ensemble import GradientBoostingClassifier #gbdt\n",
    "from sklearn import cross_validation, metrics\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "import matplotlib.pylab as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>click</th>\n",
       "      <th>C1</th>\n",
       "      <th>banner_pos</th>\n",
       "      <th>device_type</th>\n",
       "      <th>device_conn_type</th>\n",
       "      <th>C14</th>\n",
       "      <th>C15</th>\n",
       "      <th>C16</th>\n",
       "      <th>...</th>\n",
       "      <th>top_30_device_ip</th>\n",
       "      <th>top_10_device_model</th>\n",
       "      <th>top_25_device_model</th>\n",
       "      <th>top_5_device_model</th>\n",
       "      <th>top_50_device_model</th>\n",
       "      <th>top_1_device_model</th>\n",
       "      <th>top_2_device_model</th>\n",
       "      <th>top_15_device_model</th>\n",
       "      <th>top_20_device_model</th>\n",
       "      <th>top_30_device_model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>186848</td>\n",
       "      <td>1.819859e+19</td>\n",
       "      <td>0</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>20633</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>815809</td>\n",
       "      <td>3.357563e+18</td>\n",
       "      <td>0</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>15706</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>51654</td>\n",
       "      <td>1.719468e+19</td>\n",
       "      <td>0</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>21611</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>979514</td>\n",
       "      <td>1.349057e+19</td>\n",
       "      <td>0</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>19251</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>58702</td>\n",
       "      <td>1.817488e+19</td>\n",
       "      <td>1</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>15704</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 120 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0            id  click    C1  banner_pos  device_type  \\\n",
       "0      186848  1.819859e+19      0  1005           0            1   \n",
       "1      815809  3.357563e+18      0  1005           0            1   \n",
       "2       51654  1.719468e+19      0  1005           0            1   \n",
       "3      979514  1.349057e+19      0  1005           0            1   \n",
       "4       58702  1.817488e+19      1  1005           0            1   \n",
       "\n",
       "   device_conn_type    C14  C15  C16         ...           top_30_device_ip  \\\n",
       "0                 0  20633  320   50         ...                          1   \n",
       "1                 0  15706  320   50         ...                          1   \n",
       "2                 0  21611  320   50         ...                          1   \n",
       "3                 0  19251  320   50         ...                          1   \n",
       "4                 0  15704  320   50         ...                          1   \n",
       "\n",
       "   top_10_device_model  top_25_device_model  top_5_device_model  \\\n",
       "0                    0                    0                   0   \n",
       "1                    1                    1                   1   \n",
       "2                    1                    1                   1   \n",
       "3                    1                    1                   1   \n",
       "4                    1                    1                   1   \n",
       "\n",
       "   top_50_device_model  top_1_device_model  top_2_device_model  \\\n",
       "0                    1                   0                   0   \n",
       "1                    1                   1                   1   \n",
       "2                    1                   1                   1   \n",
       "3                    1                   0                   0   \n",
       "4                    1                   1                   1   \n",
       "\n",
       "   top_15_device_model  top_20_device_model  top_30_device_model  \n",
       "0                    0                    0                    0  \n",
       "1                    1                    1                    1  \n",
       "2                    1                    1                    1  \n",
       "3                    1                    1                    1  \n",
       "4                    1                    1                    1  \n",
       "\n",
       "[5 rows x 120 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取数据\n",
    "train = pd.read_csv('./train_FE.csv')\n",
    "train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 取出X和y\n",
    "X=train.drop(['click'],axis=1)\n",
    "y = train['click']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "# 切分为测试集和训练集，比例0.3\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr =LogisticRegression()\n",
    "lr.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.8391\n",
      "AUC Score (Train): 0.492022\n"
     ]
    }
   ],
   "source": [
    "y_pred = lr.predict(X_test)\n",
    "y_prit = lr.predict_proba(X_test)[:,1]\n",
    "print \"Accuracy : %.4g\" % metrics.accuracy_score(y_test, y_pred)\n",
    "print \"AUC Score (Train): %f\" % metrics.roc_auc_score(y_test, y_prit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=0.1, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l1', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrL1 = LogisticRegression(C=0.1,penalty='l1')\n",
    "lrL1.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.8379\n",
      "AUC Score (Train): 0.682001\n"
     ]
    }
   ],
   "source": [
    "y_pred = lrL1.predict(X_test)\n",
    "y_prit = lrL1.predict_proba(X_test)[:,1]\n",
    "print \"Accuracy : %.4g\" % metrics.accuracy_score(y_test, y_pred)\n",
    "print \"AUC Score (Train): %f\" % metrics.roc_auc_score(y_test, y_prit)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "加上正则之后的效果好了很多\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 试试GBDT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.8416\n",
      "AUC Score (Train): 0.718350\n"
     ]
    }
   ],
   "source": [
    "gbm0 = GradientBoostingClassifier()\n",
    "gbm0.fit(X_train,y_train)\n",
    "y_pred = gbm0.predict(X_test)\n",
    "y_predprob = gbm0.predict_proba(X_test)[:,1]\n",
    "print \"Accuracy : %.4g\" % metrics.accuracy_score(y_test, y_pred)\n",
    "print \"AUC Score (Train): %f\" % metrics.roc_auc_score(y_test, y_predprob)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "默认的GBDT都比它LR好点"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 试试模型融合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gbdt_lr_l1(X_all,y_all):\n",
    " \n",
    "    # 训练/测试数据分割\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X_all, y_all, test_size = 0.3, random_state = 42)\n",
    "\n",
    "    # 定义GBDT模型\n",
    "    gbdt = GradientBoostingClassifier(n_estimators=40, max_depth=3, verbose=0,max_features=0.5)\n",
    "     # 定义LR模型\n",
    "    lr = LogisticRegression(C=0.1,penalty='l1')\n",
    "\n",
    "    # 训练学习\n",
    "    gbdt.fit(X_train, y_train)\n",
    "\n",
    "   \n",
    "    grd_enc = OneHotEncoder()\n",
    "    \n",
    "    # fit one-hot编码器\n",
    "    grd_enc.fit(gbdt.apply(X_train)[:, :, 0])\n",
    "\n",
    "    ''' \n",
    "    使用训练好的GBDT模型构建特征，然后将特征经过one-hot编码作为新的特征输入到LR模型训练。\n",
    "    '''\n",
    "    lr.fit(grd_enc.transform(gbdt.apply(X_train)[:, :, 0]), y_train)\n",
    "    # 用训练好的LR模型多X_test做预测\n",
    "    y_pred_grd_lm = lr.predict_proba(grd_enc.transform(gbdt.apply(X_test)[:, :, 0]))[:, 1]\n",
    "\n",
    "   \n",
    "    print \"AUC Score (Train): %f\" % metrics.roc_auc_score(y_test, y_pred_grd_lm)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score (Train): 0.723601\n"
     ]
    }
   ],
   "source": [
    "gbdt_lr_l1(X_all=X,y_all=y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "有所提升 提升了0.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
